import re
import os
import fnmatch
import gzip

def field_map(dictseq, name, func):
    for d in dictseq:
        d[name] = func(d[name])
        yield d

def gen_find(filepat,top):
    for path, dirlist, filelist in os.walk(top):
        filelist.sort()
        for name in fnmatch.filter(filelist,filepat):
            yield os.path.join(path,name)

def gen_cat(sources):
    for (filename, s) in sources:
        for item in s:
            yield (filename, item)

def gen_open(filenames):
    for name in filenames:
        basename = os.path.basename(name)
        if name.endswith(".gz"):
            yield (basename, gzip.open(name))
        elif name.endswith(".bz2"):
            yield (basename, bz2.BZ2File(name))
        else:
            yield (basename, open(name))

def lines_from_dir(filepat, dirname):
   names = gen_find(filepat,dirname)
   files = gen_open(names)
   lines = gen_cat(files)
   return lines

def lsof_log(lines):
   logpats = r'(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*' \
      '(\S+)\s*(\S+)\s*(.*)'
   logpat = re.compile(logpats)
   groups   = ((filename,logpat.match(line)) for (filename,line) in lines)
   tuples   = ((filename,g.groups()) for (filename,g) in groups if g)
   colnames = ['filename','command','pid','user','fd','type','device',
      'offset','node','name']
   log      = (dict(zip(colnames, [filename] + list(t))) for 
      (filename,t) in tuples)
   return log
